from queue import Queue
import threading
import requests
import time
import pymongo
class AQY():
    def __init__(self):
        self.client = pymongo.MongoClient(host="localhost", port=27017)
        self.collection = self.client["spiders"]["aqy1"]
        self.url = "https://mesh.if.iqiyi.com/portal/lw/videolib/data?uid=&passport_id=&ret_num=60&version=12.123.20834&device_id=969cbb0d37985c9b07cc2c6d4ea041c1&channel_id=2&page_id={}&session=&token=&os=&conduit_id=&vip=0&auth=&recent_selected_tag=&ad=%5B%7B%22lm%22%3A5%2C%22position%22%3A%22library%22%7D%5D&adExt=%7B%22r%22%3A%221.2.1-ares6-pure%22%7D&dfp=a16fc24ba13dac47e89a376ba6009eb865aa2f989071d6ea8c4ff4018a234c5586&filter=%7B%22mode%22%3A%2224%22%7D"
        self.headers = {
                "Cookie": "QC005=969cbb0d37985c9b07cc2c6d4ea041c1; QP0037=0; QC234=2d8ed44bbda53df232679afc6adda0f8; QC235=1df28289debe4b5a9bd5819b7e4f194c; QC006=92c5815d1eb19234fb8195ca13081f7b; T00404=19c5d939af7cd4acd9b8c4f74bd89a9e; curDeviceState=width%3D1912%3BconduitId%3D%3Bscale%3D100%3Bbrightness%3Ddark%3BisLowPerformPC%3D0%3Bos%3Dbrowser%3Bosv%3D10.0.19044; QC007=https%3A%2F%2Fcn.bing.com%2F; __dfp=a16fc24ba13dac47e89a376ba6009eb865aa2f989071d6ea8c4ff4018a234c5586@1735713097847@1734417098847; IMS=IggQABj_y5u7BiomCiBkZTc4NTliNmU1ZDY0ZGE3OTU3OGQxNTMyMThiNThkYxAAIgByJAogZGU3ODU5YjZlNWQ2NGRhNzk1NzhkMTUzMjE4YjU4ZGMQAIIBBCICEDWKASQKIgogZGU3ODU5YjZlNWQ2NGRhNzk1NzhkMTUzMjE4YjU4ZGM",
                "User-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
                "Authority": "mesh.if.iqiyi.com",
                "Referer": "https://www.iqiyi.com/",
                "Origin": "https://www.iqiyi.com"
        }
        self.url_queue = Queue()
        self.json_queue = Queue()
        self.con_list_queue = Queue()


    def get_url(self):
        time.sleep(2)
        for i in range(1, 5):
            self.url_queue.put(self.url.format(i))


    def get_data(self):
        while True:
            url = self.url_queue.get()
            response = requests.get(url, headers=self.headers)
            response.encoding = response.apparent_encoding
            # print(url)
            # print(response.json())
            self.json_queue.put(response.json())
            self.url_queue.task_done()


    def parse_data(self):
        while True:
            data_list = self.json_queue.get()
            video_data = data_list["data"]
            # print(video_data)
            for video in video_data:
                item = {}
                item["title"] = video["title"]
                item["desc"] = video["desc"]
                item["url"] = video["url"]
                # print(item)
                self.con_list_queue.put(item)
            self.json_queue.task_done()


    def sava_data(self):
        while True:
            item = self.con_list_queue.get()
            self.collection.insert_one(item)
            self.con_list_queue.task_done()


    def main(self):
        thread_list = []
        # 开启获取地址任务  获取url地址
        # self.get_url()
        t_url = threading.Thread(target=self.get_url)
        thread_list.append(t_url)

        # 发送请求 一般比较慢 所以用两个
        for i in range(2):
        # self.get_data()
            t_get = threading.Thread(target=self.get_data)
            thread_list.append(t_get)
        # 提取数据
        # self.parse_data()
        for i in range(2):
            t_par = threading.Thread(target=self.parse_data)
            thread_list.append(t_par)
        # self.sava_data()
        t_save = threading.Thread(target=self.sava_data)
        thread_list.append(t_save)
        for t in thread_list:
            t.daemon = True             # 守护主线程 子线程不重要
            t.start()


        for q in [self.url_queue, self.json_queue, self.con_list_queue]:
            q.join()        # 让主线程阻塞， 直到计数为0


if __name__ == "__main__":
    t1 = time.time()
    aqy = AQY()
    aqy.main()
    print("总用时：", time.time()-t1)